"""
爬取DEM文件,保存
日志生成
单线程
"""
import requests
import time
from tqdm import tqdm
from threading import Thread
import sys


def downloadResource(lat_lon):
    url = 'https://gdemdl.aster.jspacesystems.or.jp/download/Download_' + lat_lon+'.zip'
    saveUrl = './Download_Dem/'+lat_lon+'.zip'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/' +
        '537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66'
    }
    # timeout连接超时跟读取超时
    res = requests.get(url, headers=headers, timeout=(60, 60), stream=True)

    chunk_size = 1024
    content_size = int(res.headers['content-length'])  # 下载文件总大小
    size = round(content_size/1024/1024, 2)  # 单位MB

    try:
        if(res.status_code == 200):
            # 开始下载，显示下载文件大小
            print(
                'Start download,File name:[{0}],File size:[{1}] MB------------'.format(lat_lon, size))
            with open(saveUrl, 'wb') as file:
                with tqdm(res.iter_content(chunk_size=chunk_size), unit='kb') as tqdmContent:
                    for chunk in tqdmContent:
                        if chunk:
                            file.write(chunk)
            info = '[200],'+lat_lon+',size:'+str(size)+'MB\n'
            print(info)
            log(info)
            time.sleep(5)  # 成功下载,5秒后再下载下一个
        elif(res.status_code == 404):
            info = '[404],'+lat_lon+',do not exit\n'
            print(info)
            log(info)
        else:
            info = '其他[{0}],{1}'.format(res.status_code, lat_lon)
            log(info)
            # sleep(5)  # 5秒后请求其他资源
    except Exception as e:
        info = '异常:{0}\n'.format(e)
        log(info)


def traverseLonLat(lat, latRange, lon, lonRange):
    """
    lat:"N"/"S"
    latRange:list,纬度范围
    lon:"W"/"E"
    lonRange:list,经度范围
    urlList:拼接部分组成的list
    """
    urlList = []
    for latNum in range(latRange[0], latRange[1]):
        for lonNum in range(lonRange[0], lonRange[1]):
            lat_lon = getLatLonStr(lat, latNum, lon, lonNum)
            urlList.append(lat_lon)
    return urlList


def getLatLonStr(lat, latNum, lon, lonNum):
    """
    获取拼接url经纬字符串
    """
    latStr = (str(latNum)).zfill(2)
    lonStr = (str(lonNum)).zfill(3)
    return lat+latStr+lon+lonStr


def createLogTxt():
    logTxtName = time.strftime("%Y-%m-%d %H_%M_%S", time.localtime())
    global logFileUrl
    logFileUrl = './log/'+logTxtName+'.txt'
    with open(logFileUrl, "a") as f:
        f.write('log初始化\n')


def log(logInfo):
    '''
    记录日志
    '''
    timeStr = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    # 以日期来创建log文件
    with open(logFileUrl, "a") as f:
        f.write(timeStr+'-----'+logInfo)


def progress():
    # 自己写的不能显示速度
    sys.stdout.write('\rfile:[{0}],size:[{1}MB]')
    sys.stdout.flush()


if __name__ == "__main__":

    createLogTxt()

    # N00-N83,W01-W180
    urlList1 = traverseLonLat('N', [0, 84], 'W', [1, 181])
    # N00-N83,E00-W179
    urlList2 = traverseLonLat('N', [0, 84], 'E', [0, 180])
    # S01-S83,W01-W180
    urlList3 = traverseLonLat('S', [1, 84], 'W', [1, 181])
    # S01-S83,E00-E179
    urlList4 = traverseLonLat('S', [1, 84], 'E', [0, 180])

    urlList = urlList1+urlList2+urlList3+urlList4
    start = 0
    end = 20
    threadNum = 20
    print(len(urlList))
    # 这肯定不能一下子创建1W多个线程啊
    while end <= len(urlList):
        for url in urlList[start: end]:
            thread = Thread(target=downloadResource, args=(url,))
            thread.start()
        start += threadNum + 1
        end += threadNum
